{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pytorch实现两层神经网络\n",
    "- 文档：https://pytorch.org/docs/torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.1'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 用numpy实现两层神经网络\n",
    "一个全连接ReLU神经网络，一个隐藏层，没有bias。用来从x预测y，使用L2 Loss。\n",
    "- $h = W_1X$\n",
    "- $a = max(0, h)$\n",
    "- $y_{hat} = W_2a$\n",
    "\n",
    "这一实现完全使用numpy来计算前向神经网络，loss，和反向传播。\n",
    "- forward pass\n",
    "- loss\n",
    "- backward pass\n",
    "\n",
    "numpy ndarray是一个普通的n维array。它不知道任何关于深度学习或者梯度(gradient)的知识，也不知道计算图(computation graph)，只是一种用来计算数学运算的数据结构。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集维度： 1000\n",
      "itor: 0 | loss:36379916.30562085\n",
      "itor: 50 | loss:11522.84048949815\n",
      "itor: 100 | loss:312.54151381266536\n",
      "itor: 150 | loss:13.699522200570192\n",
      "itor: 200 | loss:0.7797040963869617\n",
      "itor: 250 | loss:0.05274861775962181\n",
      "itor: 300 | loss:0.004007803190526088\n",
      "itor: 350 | loss:0.00032858051495920253\n",
      "itor: 400 | loss:2.82897568570412e-05\n",
      "itor: 450 | loss:2.5142220375801123e-06\n",
      "itor: 500 | loss:2.2822002515655443e-07\n"
     ]
    }
   ],
   "source": [
    "# 样本个数，输入维度，hinton, 输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 随机创建一些训练数据\n",
    "x = np.random.randn(N, D_in)\n",
    "y = np.random.randn(N, D_out)\n",
    "print(\"训练集维度：\", len(x[1]))\n",
    "\n",
    "w1 = np.random.randn(D_in, H)\n",
    "w2 = np.random.randn(H, D_out)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for it in range(501):\n",
    "    # 向前传播\n",
    "    h = x.dot(w1)   # N * H\n",
    "    # X 与 Y 逐位比较取其大者, 至少接收两个参数\n",
    "    h_relu = np.maximum(h, 0)  # N * H\n",
    "    y_pred = h_relu.dot(w2)  # N * D_out\n",
    "    \n",
    "    # 计算损失\n",
    "    loss = np.square(y_pred - y).sum()\n",
    "    # print(it, loss)\n",
    "    \n",
    "    # 反向传播\n",
    "    # 计算梯度\n",
    "    grad_y_pred = 2.0 * (y_pred - y)  # N * D_out\n",
    "    grad_w2 = h_relu.T.dot(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.dot(w2.T)\n",
    "    grad_h = grad_h_relu.copy()\n",
    "    grad_h[h<0] = 0\n",
    "    grad_w1 = x.T.dot(grad_h)\n",
    "    \n",
    "    # update weights of w1 and w2\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2\n",
    "    if it % 50 == 0:\n",
    "        print(\"itor: {} | loss:{}\".format(it, loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: Tensors\n",
    "- 使用PyTorch tensors来创建前向神经网络，计算损失，以及反向传播。\n",
    "- 一个PyTorch Tensor很像一个numpy的ndarray。但是它和numpy ndarray最大的区别是，PyTorch Tensor可以在CPU或者GPU上运算。如果想要在GPU上运算，就需要把Tensor换成cuda类型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "itor: 0 | loss:26421104.0\n",
      "itor: 50 | loss:8897.6162109375\n",
      "itor: 100 | loss:248.85116577148438\n",
      "itor: 150 | loss:12.546610832214355\n",
      "itor: 200 | loss:0.842868447303772\n",
      "itor: 250 | loss:0.06756731867790222\n",
      "itor: 300 | loss:0.006269404664635658\n",
      "itor: 350 | loss:0.0008211490930989385\n",
      "itor: 400 | loss:0.0001843837380874902\n",
      "itor: 450 | loss:6.254202889977023e-05\n",
      "itor: 500 | loss:2.910074545070529e-05\n"
     ]
    }
   ],
   "source": [
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 随机创建一些训练数据\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "w1 = torch.randn(D_in, H)\n",
    "w2 = torch.randn(H, D_out)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for it in range(501):\n",
    "    # Forward pass\n",
    "    h = x.mm(w1) # N * H\n",
    "    # clamp(min=x)小于x的等于x，＞x等于本身\n",
    "    h_relu = h.clamp(min=0) # N * H\n",
    "    y_pred = h_relu.mm(w2) # N * D_out\n",
    "    \n",
    "    # compute loss\n",
    "    loss = (y_pred - y).pow(2).sum().item()\n",
    "    \n",
    "    # Backward pass\n",
    "    # compute the gradient\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    ## mm是点乘\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h<0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "    \n",
    "    # update weights of w1 and w2\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2\n",
    "    if it % 50 == 0:\n",
    "        print(\"itor: {} | loss:{}\".format(it, loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1,  2,  3]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.tensor([[-1,2,3]])\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2, 3]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h_relu = a.clamp(min=0)\n",
    "h_relu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: nn\n",
    "\n",
    "- 这次我们使用PyTorch中nn这个库来构建网络。\n",
    "- 用PyTorch autograd来构建计算图和计算gradients，\n",
    "- 然后PyTorch会帮我们自动计算gradient。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "itor: 0 | loss:34639348.0\n",
      "itor: 50 | loss:14032.80859375\n",
      "itor: 100 | loss:599.4075927734375\n",
      "itor: 150 | loss:46.61559295654297\n",
      "itor: 200 | loss:4.714935302734375\n",
      "itor: 250 | loss:0.5426292419433594\n",
      "itor: 300 | loss:0.06672129034996033\n",
      "itor: 350 | loss:0.00874057225883007\n",
      "itor: 400 | loss:0.0014095803489908576\n",
      "itor: 450 | loss:0.00035330350510776043\n",
      "itor: 500 | loss:0.0001314057590207085\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 随机创建一些训练数据\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H, bias=False), # w_1 * x + b_1\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out, bias=False),\n",
    ")\n",
    "\n",
    "torch.nn.init.normal_(model[0].weight)\n",
    "torch.nn.init.normal_(model[2].weight)\n",
    "\n",
    "# model = model.cuda()\n",
    "\n",
    "loss_fn = nn.MSELoss(reduction='sum')\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for it in range(501):\n",
    "    # Forward pass\n",
    "    y_pred = model(x) # model.forward() \n",
    "    \n",
    "    # compute loss\n",
    "    loss = loss_fn(y_pred, y) # computation graph\n",
    "#     print(it, loss.item())\n",
    "    \n",
    "    # Backward pass\n",
    "    loss.backward()\n",
    "    \n",
    "    # update weights of w1 and w2\n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters(): # param (tensor, grad)\n",
    "            param -= learning_rate * param.grad\n",
    "            \n",
    "    model.zero_grad()\n",
    "    if it % 50 == 0:\n",
    "        print(\"itor: {} | loss:{}\".format(it, loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: optim\n",
    "\n",
    "- 这一次我们不再手动更新模型的weights,而是使用optim这个包来帮助我们更新参数。\n",
    "- optim这个package提供了各种不同的模型优化方法，包括SGD+momentum, RMSProp, Adam等等。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "itor: 0 | loss:25604472.0\n",
      "itor: 50 | loss:13182.306640625\n",
      "itor: 100 | loss:355.7388916015625\n",
      "itor: 150 | loss:17.296062469482422\n",
      "itor: 200 | loss:1.1180250644683838\n",
      "itor: 250 | loss:0.08529026061296463\n",
      "itor: 300 | loss:0.0075076608918607235\n",
      "itor: 350 | loss:0.0009207671391777694\n",
      "itor: 400 | loss:0.0002065193111775443\n",
      "itor: 450 | loss:7.471397111658007e-05\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 随机创建一些训练数据\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H, bias=False), # w_1 * x + b_1\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out, bias=False),\n",
    ")\n",
    "\n",
    "torch.nn.init.normal_(model[0].weight)\n",
    "torch.nn.init.normal_(model[2].weight)\n",
    "\n",
    "# model = model.cuda()\n",
    "\n",
    "loss_fn = nn.MSELoss(reduction='sum')\n",
    "# learning_rate = 1e-4\n",
    "# optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for it in range(500):\n",
    "    # Forward pass\n",
    "    y_pred = model(x) # model.forward() \n",
    "    \n",
    "    # compute loss\n",
    "    loss = loss_fn(y_pred, y) # computation graph\n",
    "#     print(it, loss.item())\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    # Backward pass\n",
    "    loss.backward()\n",
    "    \n",
    "    # update model parameters\n",
    "    optimizer.step()\n",
    "    if it % 50 == 0:\n",
    "        print(\"itor: {} | loss:{}\".format(it, loss))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PyTorch: 自定义 nn Modules\n",
    "--------------------------\n",
    "\n",
    "我们可以定义一个模型，这个模型继承自nn.Module类。如果需要定义一个比Sequential模型更加复杂的模型，就需要定义nn.Module模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 733.0379028320312\n",
      "itor: 0 | loss:733.0379028320312\n",
      "1 715.0055541992188\n",
      "2 697.4888916015625\n",
      "3 680.5151977539062\n",
      "4 664.1006469726562\n",
      "5 648.2057495117188\n",
      "6 632.8078002929688\n",
      "7 617.8178100585938\n",
      "8 603.2796020507812\n",
      "9 589.1598510742188\n",
      "10 575.4701538085938\n",
      "11 562.09765625\n",
      "12 549.1077270507812\n",
      "13 536.4368286132812\n",
      "14 524.1356201171875\n",
      "15 512.1729736328125\n",
      "16 500.55474853515625\n",
      "17 489.3145446777344\n",
      "18 478.4518127441406\n",
      "19 467.8425598144531\n",
      "20 457.54962158203125\n",
      "21 447.5525817871094\n",
      "22 437.77862548828125\n",
      "23 428.2828674316406\n",
      "24 419.04010009765625\n",
      "25 410.01165771484375\n",
      "26 401.2219543457031\n",
      "27 392.6513977050781\n",
      "28 384.2953796386719\n",
      "29 376.14251708984375\n",
      "30 368.1478576660156\n",
      "31 360.3475036621094\n",
      "32 352.6905517578125\n",
      "33 345.2397766113281\n",
      "34 337.96453857421875\n",
      "35 330.8605041503906\n",
      "36 323.9277648925781\n",
      "37 317.14654541015625\n",
      "38 310.4918212890625\n",
      "39 303.96148681640625\n",
      "40 297.5679016113281\n",
      "41 291.3141174316406\n",
      "42 285.1827087402344\n",
      "43 279.1679992675781\n",
      "44 273.2625732421875\n",
      "45 267.4571228027344\n",
      "46 261.7709655761719\n",
      "47 256.1896667480469\n",
      "48 250.71438598632812\n",
      "49 245.3334503173828\n",
      "50 240.03909301757812\n",
      "itor: 50 | loss:240.03909301757812\n",
      "51 234.8412322998047\n",
      "52 229.7264404296875\n",
      "53 224.70428466796875\n",
      "54 219.77076721191406\n",
      "55 214.94232177734375\n",
      "56 210.19667053222656\n",
      "57 205.53404235839844\n",
      "58 200.95599365234375\n",
      "59 196.4488067626953\n",
      "60 192.01954650878906\n",
      "61 187.6593780517578\n",
      "62 183.37176513671875\n",
      "63 179.15797424316406\n",
      "64 175.02517700195312\n",
      "65 170.9685516357422\n",
      "66 166.99330139160156\n",
      "67 163.08091735839844\n",
      "68 159.24078369140625\n",
      "69 155.4721221923828\n",
      "70 151.7720184326172\n",
      "71 148.15354919433594\n",
      "72 144.6126251220703\n",
      "73 141.1290283203125\n",
      "74 137.70669555664062\n",
      "75 134.35499572753906\n",
      "76 131.06094360351562\n",
      "77 127.82689666748047\n",
      "78 124.64949035644531\n",
      "79 121.52757263183594\n",
      "80 118.46273040771484\n",
      "81 115.45610809326172\n",
      "82 112.50567626953125\n",
      "83 109.60916137695312\n",
      "84 106.76937866210938\n",
      "85 103.97940826416016\n",
      "86 101.2391128540039\n",
      "87 98.5517578125\n",
      "88 95.91950988769531\n",
      "89 93.336181640625\n",
      "90 90.8083267211914\n",
      "91 88.32605743408203\n",
      "92 85.8914566040039\n",
      "93 83.5086669921875\n",
      "94 81.17327117919922\n",
      "95 78.8879623413086\n",
      "96 76.6518783569336\n",
      "97 74.46295166015625\n",
      "98 72.324951171875\n",
      "99 70.23460388183594\n",
      "100 68.19005584716797\n",
      "itor: 100 | loss:68.19005584716797\n",
      "101 66.19355773925781\n",
      "102 64.2402572631836\n",
      "103 62.33251953125\n",
      "104 60.46738815307617\n",
      "105 58.646697998046875\n",
      "106 56.868186950683594\n",
      "107 55.13227844238281\n",
      "108 53.44150924682617\n",
      "109 51.79106140136719\n",
      "110 50.182125091552734\n",
      "111 48.614463806152344\n",
      "112 47.08914566040039\n",
      "113 45.599822998046875\n",
      "114 44.148719787597656\n",
      "115 42.73536682128906\n",
      "116 41.35887908935547\n",
      "117 40.017032623291016\n",
      "118 38.711524963378906\n",
      "119 37.4423828125\n",
      "120 36.207462310791016\n",
      "121 35.00621795654297\n",
      "122 33.836585998535156\n",
      "123 32.700279235839844\n",
      "124 31.596023559570312\n",
      "125 30.52574348449707\n",
      "126 29.485641479492188\n",
      "127 28.475622177124023\n",
      "128 27.494762420654297\n",
      "129 26.542402267456055\n",
      "130 25.618410110473633\n",
      "131 24.72108268737793\n",
      "132 23.8505802154541\n",
      "133 23.007007598876953\n",
      "134 22.188146591186523\n",
      "135 21.395854949951172\n",
      "136 20.62778091430664\n",
      "137 19.88361358642578\n",
      "138 19.163785934448242\n",
      "139 18.465486526489258\n",
      "140 17.789155960083008\n",
      "141 17.13450050354004\n",
      "142 16.501205444335938\n",
      "143 15.88807201385498\n",
      "144 15.295546531677246\n",
      "145 14.722426414489746\n",
      "146 14.168596267700195\n",
      "147 13.63260555267334\n",
      "148 13.114553451538086\n",
      "149 12.614130020141602\n",
      "150 12.131239891052246\n",
      "itor: 150 | loss:12.131239891052246\n",
      "151 11.66529655456543\n",
      "152 11.215197563171387\n",
      "153 10.780630111694336\n",
      "154 10.361506462097168\n",
      "155 9.956844329833984\n",
      "156 9.566601753234863\n",
      "157 9.1907320022583\n",
      "158 8.828181266784668\n",
      "159 8.478806495666504\n",
      "160 8.142232894897461\n",
      "161 7.8181891441345215\n",
      "162 7.506022930145264\n",
      "163 7.2052321434021\n",
      "164 6.91557502746582\n",
      "165 6.636617660522461\n",
      "166 6.368254661560059\n",
      "167 6.110063552856445\n",
      "168 5.8614606857299805\n",
      "169 5.622430801391602\n",
      "170 5.392721176147461\n",
      "171 5.171601295471191\n",
      "172 4.959075927734375\n",
      "173 4.7549896240234375\n",
      "174 4.559126853942871\n",
      "175 4.3705878257751465\n",
      "176 4.189553260803223\n",
      "177 4.015899658203125\n",
      "178 3.8490376472473145\n",
      "179 3.689058780670166\n",
      "180 3.5353012084960938\n",
      "181 3.387801170349121\n",
      "182 3.2462048530578613\n",
      "183 3.110319137573242\n",
      "184 2.97992205619812\n",
      "185 2.854776382446289\n",
      "186 2.734652280807495\n",
      "187 2.6196062564849854\n",
      "188 2.509150743484497\n",
      "189 2.4034101963043213\n",
      "190 2.301755905151367\n",
      "191 2.2043099403381348\n",
      "192 2.110752820968628\n",
      "193 2.021108388900757\n",
      "194 1.9352127313613892\n",
      "195 1.852871060371399\n",
      "196 1.7738977670669556\n",
      "197 1.6982849836349487\n",
      "198 1.625936508178711\n",
      "199 1.5565831661224365\n",
      "200 1.4901046752929688\n",
      "itor: 200 | loss:1.4901046752929688\n",
      "201 1.426509141921997\n",
      "202 1.3656851053237915\n",
      "203 1.307332158088684\n",
      "204 1.2515028715133667\n",
      "205 1.1980562210083008\n",
      "206 1.14688241481781\n",
      "207 1.0978437662124634\n",
      "208 1.0508701801300049\n",
      "209 1.0059775114059448\n",
      "210 0.9629369974136353\n",
      "211 0.921754002571106\n",
      "212 0.882357656955719\n",
      "213 0.8445923924446106\n",
      "214 0.8084595799446106\n",
      "215 0.7739132642745972\n",
      "216 0.740837812423706\n",
      "217 0.7091805338859558\n",
      "218 0.6788672804832458\n",
      "219 0.6498571038246155\n",
      "220 0.6220843195915222\n",
      "221 0.5954957008361816\n",
      "222 0.5700555443763733\n",
      "223 0.5457085371017456\n",
      "224 0.5224031805992126\n",
      "225 0.5000970363616943\n",
      "226 0.47873106598854065\n",
      "227 0.45830944180488586\n",
      "228 0.4387199580669403\n",
      "229 0.4199889600276947\n",
      "230 0.40207639336586\n",
      "231 0.38491538166999817\n",
      "232 0.36848723888397217\n",
      "233 0.3527631163597107\n",
      "234 0.33773520588874817\n",
      "235 0.3233593702316284\n",
      "236 0.3095877170562744\n",
      "237 0.2964196503162384\n",
      "238 0.28382357954978943\n",
      "239 0.2717515230178833\n",
      "240 0.26019781827926636\n",
      "241 0.24913620948791504\n",
      "242 0.2385556846857071\n",
      "243 0.22840794920921326\n",
      "244 0.21870805323123932\n",
      "245 0.20940563082695007\n",
      "246 0.20050184428691864\n",
      "247 0.19198377430438995\n",
      "248 0.1838187426328659\n",
      "249 0.17600645124912262\n",
      "250 0.16852475702762604\n",
      "itor: 250 | loss:0.16852475702762604\n",
      "251 0.16135680675506592\n",
      "252 0.15449529886245728\n",
      "253 0.14792171120643616\n",
      "254 0.14162664115428925\n",
      "255 0.13559795916080475\n",
      "256 0.12982532382011414\n",
      "257 0.12429676204919815\n",
      "258 0.11900199949741364\n",
      "259 0.11392949521541595\n",
      "260 0.10907794535160065\n",
      "261 0.10443369299173355\n",
      "262 0.09998496621847153\n",
      "263 0.0957256481051445\n",
      "264 0.09164328873157501\n",
      "265 0.087736114859581\n",
      "266 0.08399540185928345\n",
      "267 0.08040788024663925\n",
      "268 0.07697564363479614\n",
      "269 0.07368747144937515\n",
      "270 0.07053933292627335\n",
      "271 0.06752251833677292\n",
      "272 0.06463374197483063\n",
      "273 0.061865806579589844\n",
      "274 0.05921589583158493\n",
      "275 0.05668210983276367\n",
      "276 0.0542558878660202\n",
      "277 0.05193295702338219\n",
      "278 0.04970851540565491\n",
      "279 0.0475761853158474\n",
      "280 0.04553404077887535\n",
      "281 0.04357927292585373\n",
      "282 0.041706398129463196\n",
      "283 0.0399136021733284\n",
      "284 0.03819454833865166\n",
      "285 0.036549121141433716\n",
      "286 0.03497181087732315\n",
      "287 0.0334622897207737\n",
      "288 0.032016571611166\n",
      "289 0.030631322413682938\n",
      "290 0.02930576354265213\n",
      "291 0.02803533524274826\n",
      "292 0.026819124817848206\n",
      "293 0.025654619559645653\n",
      "294 0.024539165198802948\n",
      "295 0.02347114495933056\n",
      "296 0.022448502480983734\n",
      "297 0.02146925777196884\n",
      "298 0.020531175658106804\n",
      "299 0.01963333785533905\n",
      "300 0.01877356879413128\n",
      "itor: 300 | loss:0.01877356879413128\n",
      "301 0.017950711771845818\n",
      "302 0.01716262847185135\n",
      "303 0.016408631578087807\n",
      "304 0.01568690501153469\n",
      "305 0.01499559823423624\n",
      "306 0.014334158971905708\n",
      "307 0.013700740411877632\n",
      "308 0.013094576075673103\n",
      "309 0.012514401227235794\n",
      "310 0.011959209106862545\n",
      "311 0.011427808552980423\n",
      "312 0.010919475927948952\n",
      "313 0.010433000512421131\n",
      "314 0.009967455640435219\n",
      "315 0.00952212419360876\n",
      "316 0.009096047841012478\n",
      "317 0.008688473142683506\n",
      "318 0.008298458531498909\n",
      "319 0.007925556041300297\n",
      "320 0.00756881246343255\n",
      "321 0.007227570749819279\n",
      "322 0.006901302374899387\n",
      "323 0.00658924225717783\n",
      "324 0.00629085348919034\n",
      "325 0.006005675531923771\n",
      "326 0.005732644349336624\n",
      "327 0.005471854005008936\n",
      "328 0.005222533363848925\n",
      "329 0.004984118975698948\n",
      "330 0.0047562927938997746\n",
      "331 0.004538547247648239\n",
      "332 0.004330404102802277\n",
      "333 0.004131576512008905\n",
      "334 0.003941478207707405\n",
      "335 0.0037598349153995514\n",
      "336 0.0035863355733454227\n",
      "337 0.0034205319825559855\n",
      "338 0.0032621929422020912\n",
      "339 0.003110914258286357\n",
      "340 0.0029663697350770235\n",
      "341 0.0028283866122365\n",
      "342 0.0026965574361383915\n",
      "343 0.0025706575252115726\n",
      "344 0.0024504426401108503\n",
      "345 0.002335740951821208\n",
      "346 0.002226143376901746\n",
      "347 0.002121537923812866\n",
      "348 0.0020216854754835367\n",
      "349 0.0019263827707618475\n",
      "350 0.0018354194471612573\n",
      "itor: 350 | loss:0.0018354194471612573\n",
      "351 0.0017486372962594032\n",
      "352 0.0016657868400216103\n",
      "353 0.0015867657493799925\n",
      "354 0.0015113611007109284\n",
      "355 0.0014393865130841732\n",
      "356 0.0013707540929317474\n",
      "357 0.00130530446767807\n",
      "358 0.0012428824556991458\n",
      "359 0.0011833207681775093\n",
      "360 0.001126525574363768\n",
      "361 0.0010723703308030963\n",
      "362 0.0010207598097622395\n",
      "363 0.000971512752585113\n",
      "364 0.0009245909750461578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "365 0.0008798656053841114\n",
      "366 0.0008372166194021702\n",
      "367 0.0007965742261148989\n",
      "368 0.0007578626973554492\n",
      "369 0.0007209571776911616\n",
      "370 0.0006857806001789868\n",
      "371 0.000652278249617666\n",
      "372 0.0006203744560480118\n",
      "373 0.0005900075775571167\n",
      "374 0.0005610770895145833\n",
      "375 0.0005335356690920889\n",
      "376 0.0005072933272458613\n",
      "377 0.0004823018389288336\n",
      "378 0.00045850162860006094\n",
      "379 0.0004358526784926653\n",
      "380 0.00041428091935813427\n",
      "381 0.00039375151391141117\n",
      "382 0.00037419970612972975\n",
      "383 0.0003555973235052079\n",
      "384 0.00033788266591727734\n",
      "385 0.0003210420545656234\n",
      "386 0.0003050037776120007\n",
      "387 0.00028973701409995556\n",
      "388 0.0002752211585175246\n",
      "389 0.0002614006807561964\n",
      "390 0.0002482580894138664\n",
      "391 0.00023575864906888455\n",
      "392 0.0002238633023807779\n",
      "393 0.00021254907187540084\n",
      "394 0.000201794522581622\n",
      "395 0.0001915686298161745\n",
      "396 0.00018184821237809956\n",
      "397 0.00017260605818592012\n",
      "398 0.0001638112444197759\n",
      "399 0.00015545614587608725\n",
      "400 0.0001475143217248842\n",
      "itor: 400 | loss:0.0001475143217248842\n",
      "401 0.0001399632601533085\n",
      "402 0.00013278757978696376\n",
      "403 0.00012597277236636728\n",
      "404 0.00011950647603953257\n",
      "405 0.0001133719488279894\n",
      "406 0.00010754159302450716\n",
      "407 0.00010200327960774302\n",
      "408 9.674495231593028e-05\n",
      "409 9.175214654533193e-05\n",
      "410 8.700854959897697e-05\n",
      "411 8.250032260548323e-05\n",
      "412 7.822456245776266e-05\n",
      "413 7.416475273203105e-05\n",
      "414 7.030786946415901e-05\n",
      "415 6.664611282758415e-05\n",
      "416 6.317067163763568e-05\n",
      "417 5.987334589008242e-05\n",
      "418 5.673985288012773e-05\n",
      "419 5.376888293540105e-05\n",
      "420 5.094870357424952e-05\n",
      "421 4.82736541016493e-05\n",
      "422 4.5731492718914524e-05\n",
      "423 4.33214008808136e-05\n",
      "424 4.1036022594198585e-05\n",
      "425 3.886507329298183e-05\n",
      "426 3.680891313706525e-05\n",
      "427 3.485591150820255e-05\n",
      "428 3.300643220427446e-05\n",
      "429 3.124899376416579e-05\n",
      "430 2.958733239211142e-05\n",
      "431 2.800888250931166e-05\n",
      "432 2.6511224859859794e-05\n",
      "433 2.509452315280214e-05\n",
      "434 2.3747425075271167e-05\n",
      "435 2.2474763682112098e-05\n",
      "436 2.126479557773564e-05\n",
      "437 2.012101685977541e-05\n",
      "438 1.9035418517887592e-05\n",
      "439 1.8007665858021937e-05\n",
      "440 1.7034471966326237e-05\n",
      "441 1.6110379874589853e-05\n",
      "442 1.5236693798215128e-05\n",
      "443 1.440924188500503e-05\n",
      "444 1.362529019388603e-05\n",
      "445 1.2881573638878763e-05\n",
      "446 1.2177484677522443e-05\n",
      "447 1.151292963186279e-05\n",
      "448 1.0881620255531743e-05\n",
      "449 1.0284459676768165e-05\n",
      "450 9.72040925262263e-06\n",
      "itor: 450 | loss:9.72040925262263e-06\n",
      "451 9.185148883261718e-06\n",
      "452 8.679205166117754e-06\n",
      "453 8.199809599318542e-06\n",
      "454 7.74587897467427e-06\n",
      "455 7.317171366594266e-06\n",
      "456 6.912358458066592e-06\n",
      "457 6.5278377405775245e-06\n",
      "458 6.164257683849428e-06\n",
      "459 5.820666501676897e-06\n",
      "460 5.496231551660458e-06\n",
      "461 5.188785962673137e-06\n",
      "462 4.89842977913213e-06\n",
      "463 4.624025223165518e-06\n",
      "464 4.36335540143773e-06\n",
      "465 4.118970082345186e-06\n",
      "466 3.886555987264728e-06\n",
      "467 3.6669953260570765e-06\n",
      "468 3.4603658605192322e-06\n",
      "469 3.2640502922731685e-06\n",
      "470 3.079132966377074e-06\n",
      "471 2.9048217129457043e-06\n",
      "472 2.7397129542805487e-06\n",
      "473 2.582770321168937e-06\n",
      "474 2.436055183352437e-06\n",
      "475 2.297183073096676e-06\n",
      "476 2.1656144326698268e-06\n",
      "477 2.040970230154926e-06\n",
      "478 1.9233864350098884e-06\n",
      "479 1.812603727557871e-06\n",
      "480 1.7083531247408246e-06\n",
      "481 1.6101266737678088e-06\n",
      "482 1.5170415963439154e-06\n",
      "483 1.4289674936662777e-06\n",
      "484 1.3461143453241675e-06\n",
      "485 1.2679073506660643e-06\n",
      "486 1.194136189042183e-06\n",
      "487 1.124579057432129e-06\n",
      "488 1.05921310478152e-06\n",
      "489 9.969859320335672e-07\n",
      "490 9.3883414820084e-07\n",
      "491 8.835854146127531e-07\n",
      "492 8.318909294757759e-07\n",
      "493 7.828302841517143e-07\n",
      "494 7.369587819994194e-07\n",
      "495 6.932365295142517e-07\n",
      "496 6.522329840663588e-07\n",
      "497 6.13299448559701e-07\n",
      "498 5.771529458797886e-07\n",
      "499 5.429189400274481e-07\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 随机创建一些训练数据\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        # define the model architecture\n",
    "        self.linear1 = torch.nn.Linear(D_in, H, bias=False)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out, bias=False)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        y_pred = self.linear2(self.linear1(x).clamp(min=0))\n",
    "        return y_pred\n",
    "\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "loss_fn = nn.MSELoss(reduction='sum')\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for it in range(500):\n",
    "    # Forward pass\n",
    "    y_pred = model(x) # model.forward() \n",
    "    \n",
    "    # compute loss\n",
    "    loss = loss_fn(y_pred, y) # computation graph\n",
    "    print(it, loss.item())\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    # Backward pass\n",
    "    loss.backward()\n",
    "    \n",
    "    # update model parameters\n",
    "    optimizer.step()\n",
    "    if it % 50 == 0:\n",
    "        print(\"itor: {} | loss:{}\".format(it, loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
